import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.xerces.impl.xpath.XPath;
import org.apache.xml.dtm.ref.DTMNodeList;
import org.apache.xpath.NodeSet;
import org.jsoup.nodes.Attribute;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

public class ChonCrawler {

	private String baseUrl;
	private ArrayList<ChonProduct> productToBeCheck = new ArrayList<ChonProduct>();
	private ArrayList<ChonProduct> productToBeWrite = new ArrayList<ChonProduct>();

	public ChonCrawler(String baseUrl) {
		super();
		this.baseUrl = baseUrl;
	}

	public void crawl() throws IOException, InterruptedException {
		File file = new File("D:\\ChonProduct-New.xml");
		if (!checkFileExisted(file))
			createBaseXMLFile(file);
		for (String categoryUrl : getParentCategoryList()) {
			List<String> subCategoryUrls = getSubCateogry(categoryUrl);

			ExecutorService pool = Executors.newFixedThreadPool(2);
			ChildCategoryInfo xThread;

			for (String subCategory : subCategoryUrls) {
				xThread = this.new ChildCategoryInfo(subCategory);
				pool.execute(xThread);
				xThread.join();
			}

			while (!pool.isTerminated()) {
				pool.shutdown();
			}
		}
	}

	public List<String> getParentCategoryList() throws IOException {
		List<String> categoryList = new ArrayList<String>();
		Document doc = Crawler.getHtml(baseUrl);

		Element menu = doc.getElementById("liProductlist");
		Elements subCategories = menu.getElementsByTag("ul").get(0)
				.getElementsByTag("li");
		for (Element element : subCategories) {
			categoryList.add(element.getElementsByTag("a").get(0).attr("href"));
		}

		return categoryList;
	}

	private List<String> getSubCateogry(String categoryUrl) {
		List<String> subCategoryUrl = new ArrayList<String>();

		Document doc = Crawler.getHtml(baseUrl + categoryUrl);
		for (Element element : doc.getElementsByClass("cat_child").get(0)
				.getElementsByTag("a")) {
			subCategoryUrl.add(element.attr("href"));
		}
		return subCategoryUrl;
	}

	private boolean checkFileExisted(File file) {
		// TODO Auto-generated method stub
		if (file.exists()) {
			try {
				DocumentBuilderFactory factory = DocumentBuilderFactory
						.newInstance();
				DocumentBuilder builder = factory.newDocumentBuilder();
				org.w3c.dom.Document doc = builder.parse(new FileInputStream(
						file));
				XPathFactory xPathfactory = XPathFactory.newInstance();
				javax.xml.xpath.XPath xpath = xPathfactory.newXPath();
				XPathExpression expr = xpath.compile("//ProductList");
				Node test = (Node) expr.evaluate(doc, XPathConstants.NODE);
				if (test == null) {
					return false;
				}
				if (test.getNodeName().equals("ProductList")) {
					return true;
				} else {
					return false;
				}

			} catch (XPathExpressionException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				return false;
			} catch (ParserConfigurationException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				return false;
			} catch (SAXException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				return false;
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
				return false;
			}

		} else {
			return false;
		}
	}

	private void createBaseXMLFile(File file) {
		// TODO Auto-generated method stub
		try {

			DocumentBuilderFactory factory = DocumentBuilderFactory
					.newInstance();
			DocumentBuilder builder = factory.newDocumentBuilder();
			org.w3c.dom.Document doc = builder.newDocument();

			org.w3c.dom.Element rootElement = doc.createElement("Chon.vn");
			doc.appendChild(rootElement);

			org.w3c.dom.Element productList = doc.createElement("ProductList");
			rootElement.appendChild(productList);

			TransformerFactory transformerFactory = TransformerFactory
					.newInstance();
			Transformer transformer = transformerFactory.newTransformer();
			DOMSource source = new DOMSource(doc);
			StreamResult result = new StreamResult(file);

			transformer.transform(source, result);

		} catch (ParserConfigurationException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (TransformerConfigurationException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (TransformerException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	private synchronized org.w3c.dom.Document processXMLFile(
			org.w3c.dom.Document doc, File file) {
		// TODO Auto-generated method stub
		if (doc == null) {
			System.out.println("Start Reading File");
			DocumentBuilderFactory factory = DocumentBuilderFactory
					.newInstance();
			DocumentBuilder builder;
			try {
				builder = factory.newDocumentBuilder();
				doc = builder.parse(new FileInputStream(file));
			} catch (ParserConfigurationException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (FileNotFoundException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (SAXException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			System.out.println("Finish Reading File");
			return doc;
		} else {
			WriteXmlFile(doc, file);
		}

		return null;
	}

	private synchronized void WriteXmlFile(org.w3c.dom.Document doc, File file) {
		// TODO Auto-generated method stub
		TransformerFactory transformerFactory = TransformerFactory
				.newInstance();
		Transformer transformer;
		System.out.println("Start Wrting File");
		try {
			transformer = transformerFactory.newTransformer();
			transformer.setOutputProperty(OutputKeys.INDENT, "yes");
			transformer.setOutputProperty(OutputKeys.METHOD, "xml");
			transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
			transformer.setOutputProperty(
					"{http://xml.apache.org/xslt}indent-amount", "4");
			DOMSource source = new DOMSource(doc);
			StreamResult result = new StreamResult(file);

			transformer.transform(source, result);
		} catch (TransformerConfigurationException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (TransformerException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		System.out.println("Finish Writing File");
	}

	class ChildCategoryInfo extends Thread {
		String subCategory;

		public ChildCategoryInfo(String subCategory) {
			this.subCategory = subCategory;
		}

		@Override
		public void run() {
			int pageNumber;
			try {
				Document doc = Crawler.getHtml(baseUrl + subCategory);
				Elements navigate = null;

				navigate = doc.select("ul[class=pagging_nav] a");
				System.out.println(baseUrl + subCategory);

				String lastPageUrl = navigate.get(navigate.size() - 1).attr(
						"href");

				pageNumber = Integer.parseInt(lastPageUrl.substring(lastPageUrl
						.indexOf("page=") + 5));
				System.out.println(pageNumber);
			} catch (Exception ex) {
				System.out.println(baseUrl + subCategory + "-"
						+ ex.getMessage());
				pageNumber = 1;
			}

			ExecutorService pool = Executors.newFixedThreadPool(3);
			PageInfo pageInfo;

			int index = 1;
			while (index <= pageNumber) {
				pageInfo = new PageInfo(baseUrl + subCategory
						+ "sort=lastestproduct&page=" + index);
				pool.execute(pageInfo);
				try {
					pageInfo.join();
				} catch (InterruptedException e) {
					e.printStackTrace();
				}
				index++;
			}

			while (!pool.isTerminated()) {
				pool.shutdown();
			}

			System.out.println("Thread Ok - " + subCategory + " - " + index);

			return;
		}
	}

	public synchronized void checkWaitingProduct(
			ArrayList<ChonProduct> productToBeCheck) {
		File file = new File("D:\\ChonProduct-New.xml");
		System.out.println("Start check ---" + productToBeCheck.size());
		org.w3c.dom.Document doc = processXMLFile(null, file);
		ChonProduct existProduct;
		for (ChonProduct currentProduct : productToBeCheck) {
			try {
				existProduct = getCompareProduct(currentProduct.getRelId(), doc);
				if (existProduct == null) {
					productToBeWrite.add(currentProduct);
				} else if (!currentProduct.equals(existProduct)) {
					System.out.println("Update Current Product");
					// productToBeWrite.add(currentProduct);
				} else {
				}
			} catch (FileNotFoundException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (XPathExpressionException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (ParserConfigurationException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (SAXException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}

		if (productToBeWrite.size() >= 20) {
			try {
				System.out.println("Start write ---" + productToBeWrite.size());
				writeProductXml(productToBeWrite);
				productToBeWrite.clear();
			} catch (ParserConfigurationException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (TransformerException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (SAXException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}

	}

	private ChonProduct getCompareProduct(String relId, org.w3c.dom.Document doc)
			throws ParserConfigurationException, FileNotFoundException,
			SAXException, IOException, XPathExpressionException {

		XPathFactory xPathfactory = XPathFactory.newInstance();
		javax.xml.xpath.XPath xpath = xPathfactory.newXPath();

		ChonProduct existProduct = new ChonProduct();

		existProduct.setRelId(xpath.evaluate(
				"//Chon.vn/ProductList/Product[RelId='" + relId + "']/RelId",
				doc).trim());
		existProduct.setDiscount(xpath
				.evaluate(
						"//Chon.vn/ProductList/Product[RelId='" + relId
								+ "']/Discount", doc).trim());
		existProduct.setLink(xpath.evaluate(
				"//Chon.vn/ProductList/Product[RelId='" + relId + "']/Link",
				doc).trim());
		existProduct.setMsrp(xpath.evaluate(
				"//Chon.vn/ProductList/Product[RelId='" + relId + "']/MSRP",
				doc).trim());
		existProduct.setRetailPrice(xpath.evaluate(
				"//Chon.vn/ProductList/Product[RelId='" + relId
						+ "']/RetailPrice", doc).trim());
		DTMNodeList images = (DTMNodeList) xpath.evaluate(
				"//Chon.vn/ProductList/Product[RelId='" + relId
						+ "']/Images/Image", doc, XPathConstants.NODESET);
		for (int i = 0; i < images.getLength(); i++) {
			Node image = images.item(i);
			existProduct.getImages().add(image.getTextContent().trim());
		}

		if (existProduct.getRelId() == "") {
			return null;
		} else {
			return existProduct;
		}
	}

	private void writeProductXml(ArrayList<ChonProduct> productToBeWrite)
			throws ParserConfigurationException, TransformerException,
			SAXException, IOException {
		// TODO Auto-generated method stub

		File file = new File("D:\\ChonProduct-New.xml");
		org.w3c.dom.Document doc = processXMLFile(null, file);
		XPathFactory xPathfactory = XPathFactory.newInstance();
		javax.xml.xpath.XPath xpath = xPathfactory.newXPath();

		try {
			Node productList = (Node) xpath.evaluate("//ProductList", doc,
					XPathConstants.NODE);
			for (ChonProduct currentProduct : productToBeWrite) {
				productList.appendChild(doc.importNode(
						createProductXml(currentProduct), true));
			}
		} catch (XPathExpressionException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		WriteXmlFile(doc, file);

		return;
	}

	private Node createProductXml(ChonProduct currentProduct)
			throws ParserConfigurationException {

		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
		DocumentBuilder builder = factory.newDocumentBuilder();
		org.w3c.dom.Document doc = builder.newDocument();

		Node xmlProduct = doc.createElement("Product");

		org.w3c.dom.Element relId = doc.createElement("RelId");
		relId.appendChild(doc.createTextNode(currentProduct.getRelId()));

		org.w3c.dom.Element discount = doc.createElement("Discount");
		discount.appendChild(doc.createTextNode(currentProduct.getDiscount()));

		org.w3c.dom.Element link = doc.createElement("Link");
		link.appendChild(doc.createTextNode(currentProduct.getLink()));

		org.w3c.dom.Element msrp = doc.createElement("MSRP");
		msrp.appendChild(doc.createTextNode(currentProduct.getMsrp()));

		org.w3c.dom.Element retailPrice = doc.createElement("RetailPrice");
		retailPrice.appendChild(doc.createTextNode(currentProduct
				.getRetailPrice()));

		org.w3c.dom.Element images = doc.createElement("Images");
		for (String imageUrl : currentProduct.getImages()) {
			org.w3c.dom.Element image = doc.createElement("Image");
			image.appendChild(doc.createTextNode(imageUrl));
			images.appendChild(image);
		}

		xmlProduct.appendChild(relId);
		xmlProduct.appendChild(discount);
		xmlProduct.appendChild(link);
		xmlProduct.appendChild(msrp);
		xmlProduct.appendChild(retailPrice);
		xmlProduct.appendChild(images);

		return xmlProduct;
	}

	class PageInfo extends Thread {
		String pageUrl;

		public PageInfo(String pageUrl) {
			this.pageUrl = pageUrl;
		}

		@Override
		public void run() {
			Document doc = Crawler.getHtml(pageUrl);

			Elements productlist = doc
					.select("ul#list_product > li[mobivi_c=true]");

			SaveProductList(productlist);

			System.out.println(pageUrl + "-" + productlist.size());

			return;
		}

		private void SaveProductList(Elements productlist) {

			ExecutorService pool = Executors.newFixedThreadPool(3);
			ProductInfo productInfo;
			for (Element product : productlist) {
				productInfo = new ProductInfo(product);
				pool.execute(productInfo);
				try {
					productInfo.join();
				} catch (InterruptedException e) {
					e.printStackTrace();
				}
			}

			while (!pool.isTerminated()) {
				pool.shutdown();
			}
		}
	}

	class ProductInfo extends Thread {
		Element product;
		ChonProduct currentProduct;

		public ProductInfo(Element product) {
			this.product = product;
		}

		@Override
		public void run() {
			savingProductInformation();
		}

		private void savingProductInformation() {

			Elements selectResult;
			currentProduct = new ChonProduct();

			currentProduct.setRelId(product.select("[relid]").attr("relid"));
			currentProduct.setLink(product.select(
					"a[href*=aspx?ref=search-pro]").attr("href"));
			currentProduct.setDiscount(product.attr("mobivi_r"));
			currentProduct.setRetailPrice(getPriceValue(product.select(
					"[class^=price]").text()));

			selectResult = product.select("[class^=old_price]");
			if (selectResult.size() > 0) {
				currentProduct.setMsrp(getPriceValue(selectResult.text()));
			} else {
				currentProduct.setMsrp(currentProduct.getRetailPrice());
			}

			selectResult = product.select("img");

			for (Attribute attr : selectResult.get(0).attributes()) {
				if (attr.getValue().matches(".*url=.*")) {
					String attrValue = attr.getValue();
					String urlImage = attrValue.substring(attrValue
							.indexOf("url=") + 4);
					if (!currentProduct.getImages().contains(urlImage)) {
						currentProduct.getImages().add(urlImage);
					}
				}
			}
			putInToCheckList(currentProduct);
			return;
		}

		private void putInToCheckList(ChonProduct currentProduct) {
			// TODO Auto-generated method stub
			productToBeCheck.add(currentProduct);
			if (productToBeCheck.size() >= 20) {

				ArrayList<ChonProduct> transferList = (ArrayList<ChonProduct>) productToBeCheck
						.clone();
				productToBeCheck.clear();
				checkWaitingProduct(transferList);
			}
		}

		private String getPriceValue(String inputPrice) {
			// TODO Auto-generated method stub
			inputPrice = inputPrice.replaceAll("[^\\d]", "");
			return inputPrice;
		}
	}
}
